# Updated by Wei on 20140717
# make the clueweb09B results fixed and tight
# Updated by Wei on 20140712
# make the gov2 / clueweb09 work

from __future__ import division
from operator import itemgetter, attrgetter
from struct import *
import gc
import math
# import matplotlib
import os
# import pylab
import random
import sys
import time

print "Begins..."

print "len(sys.argv):",len(sys.argv)
if len(sys.argv) != 3:
    print "Illegal # of arguments"
    print "Usage: python programName.py logFile theBaseFileName"
    exit(1)
else:
    inputFileName1 = sys.argv[1]
    outputBaseFileName = sys.argv[2]
    
docIDANDTrecIDDict = {}

# for gov2
# for moa
# (NOT USED)inputFileName0 = "/home/diaosi/gov2ClearYourMindAndDoItAgain2014/trecID_docID_MappingTableForGov2Dataset"
# for vidaserver1
# inputFileName0 = "/home/vgc/wei/workspace/NYU_IRTK/data/trecID_docID_MappingTableForGov2Dataset"

# for clueweb09B
# for vidaserver1
inputFileName0 = "/local_scratch/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/clueweb09BDocumentOverallStatistics_sortedByDocID_20140606"

inputFileHandler = open(inputFileName0,"r")
currentLine = inputFileHandler.readline()
numOfDocs = 1
while currentLine:
    currentLineElements = currentLine.strip().split(" ")
    # TODO: unify them
    # format might change across different data sets
    # for clueweb09B
    currentDocID = currentLineElements[0]
    currentTrecID = currentLineElements[2]
    # for gov2
    #currentDocID = currentLineElements[1]
    #currentTrecID = currentLineElements[0]
        
    if currentDocID not in docIDANDTrecIDDict:
        docIDANDTrecIDDict[currentDocID] = currentTrecID
    '''
    if numOfDocs % 10000 == 0:
        sys.stdout.write("Docs processed: %d   \r" % (numOfDocs) )
        sys.stdout.flush()
    '''
    currentLine = inputFileHandler.readline()
    numOfDocs += 1
print "len(docIDANDTrecIDDict):",len(docIDANDTrecIDDict)
print "docIDANDTrecIDDict['0']:",docIDANDTrecIDDict['0']
inputFileHandler.close()


# for gov2
# for moa
# AND
# weight == 50
# inputFileName1 = "/home/diaosi/workspace/web-search-engine-wei-2014-April/results/evaluate_quality_weight_50_AND_20140521_2DTableComputeFromTOP100Postings"
# weight == 0
# inputFileName1 = "/home/diaosi/workspace/web-search-engine-wei-2014-April/results/evaluate_quality_weight_0_AND_20140522_2DTableComputeFromTOP10Postings"
# weight == 50, more fine grained
# inputFileName1 = "/home/diaosi/workspace/web-search-engine-wei-2014-April/results/evaluate_quality_weight_50_AND_20140529_2DTableComputeFromTOP100Postings_more_fine_grained_PART_OF"
# inputFileName1 = "/home/diaosi/workspace/web-search-engine-wei-2014-April/results/evaluate_quality_weight_10_AND_20140529_2DTableComputeFromTOP100Postings_more_fine_grained_PART_OF"
# inputFileName1 = "/home/diaosi/workspace/web-search-engine-wei-2014-April/scripts/src/pythonScripts/pruning/evaluationScripts/debug_try_20140605_PART_OF_AND"
# OR
# weight == 50
# inputFileName1 = "/home/diaosi/workspace/web-search-engine-wei-2014-April/results/evaluate_quality_weight_50_OR_20140521_2DTableComputeFromTOP100Postings"
# weight == 0
# inputFileName1 = "/home/diaosi/workspace/web-search-engine-wei-2014-April/results/evaluate_quality_weight_0_OR_20140522_2DTableComputeFromTOP10Postings"
# inputFileName1 = "/home/diaosi/workspace/web-search-engine-wei-2014-April/results/evaluate_quality_weight_50_OR_20140529_2DTableComputeFromTOP100Postings_more_fine_grained_PART_OF"
# inputFileName1 = "/home/diaosi/workspace/web-search-engine-wei-2014-April/results/"
# inputFileName1 = "/home/diaosi/workspace/web-search-engine-wei-2014-April/results/evaluate_quality_weight_10_OR_20140529_2DTableComputeFromTOP100Postings_more_fine_grained_PART_OF"
# inputFileName1 = "/home/diaosi/workspace/web-search-engine-wei-2014-April/scripts/src/pythonScripts/pruning/evaluationScripts/debug_try_20140605_PART_OF_OR"

# for clueweb09B
# for vidaserver1
indexKeptList = ["0.08%","1%","2%","3%","4%","5%","6%","7%","8%","9%","10%","15%","20%","30%","40%","50%","60%","70%","80%","90%","100%"]
# evaluateQuality_clueweb09B_static_20140626_withStopword_LOG
# evaluateQuality_clueweb09B_1_20140626_withoutStopword_LOG

# comment out for arg input
# inputFileName1 = "/local_scratch/wei/workspace/NYU_IRTK/results/dynamicUnigramFromWei/clueweb09B/evaluateQuality_clueweb09B_20_20140626_withStopword_LOG"
inputFileHandler = open(inputFileName1,"r")
# comment out for arg input
# outputBaseFileName = "/local_scratch/wei/workspace/NYU_IRTK/results/dynamicUnigramFromWei/clueweb09B/trecCompatibleResults/ourApproach_20_withoutStopWords/trecCompatible_clueweb09B_20_withoutStopWords_20140626_"
percentageIndex = 0
startToRecordTag = True
endToRecordTag = True

for line in inputFileHandler.readlines():
    lineElements = line.strip().split(" ")
    
    if len(lineElements) == 7 and lineElements[-2] == "NYU_IRTK":
        print lineElements
        percentageIndex = int(lineElements[-1])
        if startToRecordTag:
            # print "new pruning level begins..."
            outputFileName = outputBaseFileName + "_" + str(indexKeptList[percentageIndex])
            outputFileHandler = open(outputFileName,"w")
            startToRecordTag = False
        # print line.strip()
        
        currentDocID = lineElements[2]
        outputLine = lineElements[0] + " " + lineElements[1] + " " + docIDANDTrecIDDict[ currentDocID ] + " " + lineElements[3] + " " + lineElements[4] + " " + "NYU_IRTK" + "\n"
        outputFileHandler.write(outputLine)

    if not len(lineElements) == 7 and not startToRecordTag:
        if endToRecordTag:
            print "new pruning level ends."
            endToRecordTag = False
            # init
            startToRecordTag = True
            endToRecordTag = True
            print "OVERALL:"
            print "outputFileName:",outputFileName
            outputFileHandler.close()

inputFileHandler.close()

print "OVERALL:"
print "inputFileName0:",inputFileName0
print "inputFileName1:",inputFileName1
print "Ends."
